# -*- coding: utf-8 -*-

"""
@Datetime: 2019/3/13
@Author: Zhang Yafei
"""
import re
import sys
import time
import requests
from lxml.html import fromstring
from pyquery import PyQuery as pq
from bs4 import BeautifulSoup as bs
from scrapy.selector import Selector


headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36'}


def Timer():
    a = time.time()
    while True:
        c = time.time()
        yield time.time() - a
        a = c

# ################# start request #################
timer = Timer()
url = "https://www.python.org/"
html = requests.get(url, headers=headers).text
num = 10000
print('\n==== Python version: %s =====' % sys.version)
print('\n==== Total trials: %s =====' % num)
next(timer)

# ################# bs4 #########################
soup = bs(html, 'lxml')
for x in range(num):
    paragraphs = soup.findAll('p')
t = next(timer)
print('bs4 total time: %.1f' % t)
# ################ pyquery #######################
d = pq(html)
for x in range(num):
    paragraphs = d('p')
t = next(timer)
print('pq total time: %.1f' % t)
# ############### lxml css #########################
tree = fromstring(html)
for x in range(num):
    paragraphs = tree.cssselect('p')
t = next(timer)
print('lxml (cssselect) total time: %.1f' % t)
# ############## lxml xpath #######################
tree = fromstring(html)
for x in range(num):
    paragraphs = tree.xpath('.//p')
t = next(timer)
print('lxml (xpath) total time: %.1f' % t)
# ############### re ##########################
for x in range(num):
    paragraphs = re.findall('<[p ]>.*?</p>', html)
t = next(timer)
print('regex total time: %.1f (doesn\'t find all p)\n' % t)
# ############## scrpay xpath #################
selector = Selector(text=html)
for x in range(num):
    paragraphs = selector.xpath('.//p')
t = next(timer)
print('scrapy xpath total time: %.1f' % t)





